%load_ext autoreload
%autoreload 2Training Kalman Filter for Results
from meteo_imp.kalman.fastai import *
from meteo_imp.kalman.filter import *
from meteo_imp.utils import *
from meteo_imp.data import *
from meteo_imp.gaussian import *
from fastai.tabular.learner import *
from fastai.learner import *
from fastai.callback.all import *
from fastcore.foundation import L
from meteo_imp.kalman.fastai import show_results
import pandas as pd
import numpy as np
import torch
import random
from sklearn.decomposition import PCAreset_seed()haiB = pd.read_parquet(hai_big_path)
hai_eraB = pd.read_parquet(hai_era_big_path)list(haiB.columns)['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN']
TA - gap 12
dls_TA = imp_dataloader(haiB, hai_eraB, var_sel = 'TA', block_len=50, gap_len=12, bs=20, control_lags=[1], n_rep=5).cpu()model_TA = KalmanFilterSR.init_local_slope_pca(len(haiB.columns),len(haiB.columns), df_pca = haiB, pred_only_gap=True, use_conditional=True)save_models_TA = SaveModelsBatch(times_epoch=10)items_TA = random.choices(dls_TA.valid.items, k=4)learn_TA = Learner(dls_TA, model_TA, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_TA], metrics=rmse_gap)learn_TA.fit(1, 1e-3)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | -17.669978 | -18.692597 | 0.040505 | 11:47 |
learn_TA.recorder.plot_loss()
torch.save(learn_TA.model, "trained_4_feb_TA_gap_12_v1.pickle")learn_TAKalman Filter (7 obs, 14 state, 14 contr)
$A$
| state | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 1.0540 | 0.0263 | -0.1801 | 0.1166 | -0.0537 | 0.0176 | -0.0122 | 0.9143 | 0.0358 | 0.2069 | -0.1760 | 0.0973 | 0.0417 | -0.0181 |
| x_1 | 0.0331 | 1.0263 | -0.1854 | 0.1730 | -0.0476 | 0.0026 | -0.0084 | 0.0203 | 1.0045 | 0.2047 | -0.2114 | 0.0382 | 0.1549 | -0.0265 |
| x_2 | -0.0073 | -0.0518 | 1.1940 | -0.1817 | 0.0451 | 0.0350 | -0.0043 | -0.0032 | 0.0095 | 0.7757 | 0.1996 | -0.0706 | -0.0505 | 0.0121 |
| x_3 | 0.0022 | 0.0489 | -0.2066 | 1.1895 | -0.0348 | -0.0384 | 0.0039 | 0.0109 | -0.0043 | 0.2244 | 0.7726 | 0.0558 | 0.0648 | -0.0132 |
| x_4 | -0.0512 | -0.0249 | 0.2067 | -0.1689 | 1.0340 | -0.0274 | 0.0076 | 0.0279 | -0.0143 | -0.2281 | 0.2212 | 0.9361 | -0.0667 | 0.0287 |
| x_5 | -0.0319 | -0.0366 | 0.1644 | -0.1662 | 0.0304 | 0.9821 | 0.0050 | -0.0164 | 0.0291 | -0.2027 | 0.2111 | -0.0685 | 0.8243 | 0.0220 |
| x_6 | -0.0558 | 0.0097 | 0.0369 | -0.0054 | 0.0365 | -0.0207 | 1.0128 | -0.0391 | 0.0098 | -0.0321 | 0.0340 | -0.0274 | -0.0362 | 1.1283 |
| x_7 | 0.0175 | -0.0001 | -0.0490 | 0.0358 | -0.0187 | 0.0404 | -0.0147 | 1.0056 | -0.0110 | -0.1325 | 0.0999 | 0.0109 | 0.0650 | 0.0072 |
| x_8 | 0.0198 | 0.0140 | -0.0681 | 0.0669 | -0.0049 | 0.0069 | -0.0147 | -0.0261 | 0.9871 | -0.0827 | 0.0552 | 0.0196 | 0.0729 | 0.0038 |
| x_9 | -0.0124 | -0.0219 | 0.0507 | -0.0514 | 0.0192 | -0.0041 | 0.0071 | 0.0212 | 0.0110 | 1.0889 | -0.0715 | -0.0237 | -0.0894 | -0.0072 |
| x_10 | 0.0022 | 0.0194 | -0.0439 | 0.0492 | -0.0057 | -0.0017 | -0.0091 | -0.0213 | -0.0032 | -0.1142 | 1.0904 | 0.0206 | 0.0681 | 0.0026 |
| x_11 | -0.0269 | -0.0079 | 0.0741 | -0.0596 | 0.0211 | -0.0365 | 0.0127 | 0.0027 | 0.0129 | 0.0879 | -0.0527 | 0.9728 | -0.0868 | -0.0051 |
| x_12 | -0.0138 | -0.0181 | 0.0449 | -0.0477 | -0.0084 | -0.0033 | 0.0205 | 0.0170 | 0.0028 | 0.1311 | -0.1018 | -0.0136 | 0.9346 | -0.0122 |
| x_13 | -0.0303 | -0.0016 | 0.0642 | -0.0774 | 0.0129 | -0.0009 | 0.0150 | 0.0006 | 0.0311 | -0.0401 | 0.0387 | -0.0224 | -0.0663 | 1.0180 |
$Q$
| state | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 0.0603 | 0.0128 | 0.0115 | 0.0061 | 0.0150 | -0.0126 | 0.0103 | 0.0076 | -0.0200 | -0.0013 | -0.0074 | 0.0004 | 0.0211 | 0.0006 |
| x_1 | 0.0128 | 0.1126 | 0.0136 | -0.0148 | -0.0008 | -0.0037 | -0.0113 | 0.0081 | -0.0175 | 0.0001 | 0.0029 | 0.0102 | -0.0094 | 0.0078 |
| x_2 | 0.0115 | 0.0136 | 0.0798 | 0.0384 | -0.0597 | -0.0427 | -0.0099 | -0.0391 | 0.0153 | 0.0081 | -0.0137 | -0.0111 | 0.0390 | -0.0067 |
| x_3 | 0.0061 | -0.0148 | 0.0384 | 0.0868 | 0.0370 | 0.0523 | -0.0017 | 0.0122 | -0.0219 | 0.0202 | -0.0043 | 0.0327 | -0.0257 | 0.0129 |
| x_4 | 0.0150 | -0.0008 | -0.0597 | 0.0370 | 0.1871 | 0.0655 | 0.0114 | 0.0448 | -0.0234 | 0.0061 | 0.0197 | 0.0317 | -0.0447 | 0.0082 |
| x_5 | -0.0126 | -0.0037 | -0.0427 | 0.0523 | 0.0655 | 0.1900 | 0.0022 | 0.0155 | -0.0198 | -0.0082 | -0.0155 | 0.0184 | -0.0427 | 0.0062 |
| x_6 | 0.0103 | -0.0113 | -0.0099 | -0.0017 | 0.0114 | 0.0022 | 0.1411 | 0.0030 | -0.0052 | -0.0030 | 0.0024 | 0.0027 | -0.0025 | -0.0092 |
| x_7 | 0.0076 | 0.0081 | -0.0391 | 0.0122 | 0.0448 | 0.0155 | 0.0030 | 0.1724 | -0.0748 | 0.0415 | 0.0064 | 0.0843 | -0.0691 | 0.0358 |
| x_8 | -0.0200 | -0.0175 | 0.0153 | -0.0219 | -0.0234 | -0.0198 | -0.0052 | -0.0748 | 0.1626 | -0.0003 | -0.0328 | -0.0405 | 0.1111 | -0.0152 |
| x_9 | -0.0013 | 0.0001 | 0.0081 | 0.0202 | 0.0061 | -0.0082 | -0.0030 | 0.0415 | -0.0003 | 0.1007 | 0.0491 | -0.0237 | -0.0595 | -0.0133 |
| x_10 | -0.0074 | 0.0029 | -0.0137 | -0.0043 | 0.0197 | -0.0155 | 0.0024 | 0.0064 | -0.0328 | 0.0491 | 0.1143 | 0.0264 | -0.0264 | 0.0155 |
| x_11 | 0.0004 | 0.0102 | -0.0111 | 0.0327 | 0.0317 | 0.0184 | 0.0027 | 0.0843 | -0.0405 | -0.0237 | 0.0264 | 0.2406 | -0.0565 | 0.0455 |
| x_12 | 0.0211 | -0.0094 | 0.0390 | -0.0257 | -0.0447 | -0.0427 | -0.0025 | -0.0691 | 0.1111 | -0.0595 | -0.0264 | -0.0565 | 0.2944 | -0.0166 |
| x_13 | 0.0006 | 0.0078 | -0.0067 | 0.0129 | 0.0082 | 0.0062 | -0.0092 | 0.0358 | -0.0152 | -0.0133 | 0.0155 | 0.0455 | -0.0166 | 0.2207 |
$b$
| state | offset |
|---|---|
| x_0 | -0.0051 |
| x_1 | 0.0013 |
| x_2 | -0.0071 |
| x_3 | 0.0069 |
| x_4 | 0.0000 |
| x_5 | 0.0048 |
| x_6 | 0.0097 |
| x_7 | -0.0066 |
| x_8 | -0.0045 |
| x_9 | 0.0044 |
| x_10 | -0.0051 |
| x_11 | 0.0065 |
| x_12 | 0.0051 |
| x_13 | -0.0005 |
$H$
| variable | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| y_0 | 0.0096 | -0.1188 | 0.6133 | -0.2500 | 0.0814 | 0.0259 | 0.0046 | -0.0204 | -0.0681 | 0.1127 | -0.0955 | 0.0484 | -0.0042 | 0.0207 |
| y_1 | 0.9850 | 0.1105 | 0.2278 | -0.0157 | 0.0148 | -0.1788 | 0.0102 | 0.1131 | -0.0796 | 0.1846 | -0.1719 | 0.0587 | 0.1843 | -0.0333 |
| y_2 | 0.0415 | -0.0219 | 0.7257 | 0.6141 | -0.0440 | 0.0144 | -0.0002 | 0.0166 | 0.0696 | -0.1107 | 0.0991 | -0.0417 | -0.0550 | -0.0219 |
| y_3 | 0.0665 | 0.0416 | 0.2722 | -0.2680 | -0.2304 | -0.9431 | 0.0098 | 0.0517 | 0.0827 | -0.0350 | 0.0336 | -0.0461 | 0.0551 | -0.0349 |
| y_4 | -0.0238 | -0.0257 | -0.0034 | -0.0257 | 0.0291 | 0.0403 | 1.2371 | -0.0236 | 0.0283 | -0.0672 | 0.0391 | -0.0402 | -0.0049 | -0.1147 |
| y_5 | 0.1026 | -0.1011 | -0.2424 | 0.0548 | 1.0465 | 0.0073 | 0.0193 | -0.0573 | -0.0184 | -0.1912 | 0.2090 | 0.0438 | -0.0392 | 0.0262 |
| y_6 | -0.1024 | -1.0913 | -0.1905 | 0.2058 | 0.0868 | -0.1214 | 0.0166 | -0.0516 | 0.0307 | 0.0977 | -0.0751 | -0.0070 | -0.0340 | 0.0021 |
$R$
| variable | y_0 | y_1 | y_2 | y_3 | y_4 | y_5 | y_6 |
|---|---|---|---|---|---|---|---|
| y_0 | 0.0024 | -0.0171 | 0.0077 | 0.0078 | -0.0003 | 0.0176 | -0.0061 |
| y_1 | -0.0171 | 0.1305 | -0.0550 | -0.0581 | 0.0045 | -0.1186 | 0.0311 |
| y_2 | 0.0077 | -0.0550 | 0.0310 | 0.0102 | 0.0010 | 0.0652 | -0.0293 |
| y_3 | 0.0078 | -0.0581 | 0.0102 | 0.0699 | -0.0053 | 0.0594 | 0.0204 |
| y_4 | -0.0003 | 0.0045 | 0.0010 | -0.0053 | 0.0145 | 0.0066 | -0.0025 |
| y_5 | 0.0176 | -0.1186 | 0.0652 | 0.0594 | 0.0066 | 0.2732 | 0.0217 |
| y_6 | -0.0061 | 0.0311 | -0.0293 | 0.0204 | -0.0025 | 0.0217 | 0.1247 |
$d$
| variable | offset |
|---|---|
| y_0 | -0.0038 |
| y_1 | -0.0016 |
| y_2 | 0.0033 |
| y_3 | -0.0048 |
| y_4 | 0.0000 |
| y_5 | -0.0174 |
| y_6 | -0.0078 |
$B$
| state | c_0 | c_1 | c_2 | c_3 | c_4 | c_5 | c_6 | c_7 | c_8 | c_9 | c_10 | c_11 | c_12 | c_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 0.1167 | -0.9769 | -0.0553 | 0.0013 | 0.0001 | -0.0145 | -0.0575 | 0.1969 | 0.9707 | -0.0257 | -0.0092 | 0.0141 | 0.0467 | 0.0306 |
| x_1 | 0.1976 | 0.0029 | -0.0212 | 0.0082 | -0.0222 | -0.0687 | 0.9226 | 0.0571 | 0.0154 | -0.0553 | -0.0020 | -0.0017 | 0.0169 | -0.9740 |
| x_2 | -0.8669 | -0.0072 | -0.4643 | -0.0086 | 0.0123 | 0.1030 | 0.1404 | 0.7364 | 0.0033 | 0.4928 | 0.0606 | -0.0111 | -0.0839 | -0.1544 |
| x_3 | 0.5027 | 0.0291 | -0.8831 | -0.0053 | -0.0012 | -0.0808 | -0.0618 | -0.3491 | -0.0220 | 0.8552 | -0.0398 | 0.0016 | 0.0779 | 0.0841 |
| x_4 | -0.1806 | -0.0226 | 0.0442 | 0.1440 | 0.0033 | -0.9289 | 0.0550 | -0.1573 | 0.0184 | 0.0263 | -0.1280 | 0.0084 | 0.9363 | -0.0234 |
| x_5 | -0.1248 | -0.0215 | 0.0321 | 0.9747 | -0.0141 | 0.2016 | 0.0474 | -0.1575 | 0.0181 | 0.0381 | -0.9735 | 0.0186 | -0.1620 | -0.0088 |
| x_6 | -0.0989 | -0.0234 | 0.0083 | -0.0287 | -0.9298 | 0.0117 | 0.0258 | -0.1111 | -0.0121 | 0.0174 | 0.0214 | 0.9692 | -0.0285 | 0.0169 |
| x_7 | 0.0672 | -0.0093 | -0.0187 | 0.0061 | -0.0021 | 0.0007 | -0.0170 | 0.0515 | -0.0079 | -0.0154 | 0.0087 | -0.0062 | -0.0037 | -0.0194 |
| x_8 | 0.0901 | -0.0058 | -0.0231 | 0.0070 | 0.0006 | 0.0007 | -0.0120 | 0.0943 | -0.0096 | -0.0184 | 0.0063 | 0.0007 | 0.0153 | -0.0104 |
| x_9 | -0.0884 | 0.0108 | 0.0269 | -0.0104 | -0.0066 | -0.0051 | 0.0058 | -0.0851 | 0.0131 | 0.0219 | -0.0113 | -0.0029 | -0.0140 | 0.0054 |
| x_10 | 0.0883 | -0.0110 | -0.0225 | 0.0082 | 0.0050 | 0.0123 | -0.0054 | 0.0796 | -0.0118 | -0.0177 | 0.0095 | 0.0018 | 0.0159 | -0.0074 |
| x_11 | -0.0780 | 0.0025 | 0.0207 | -0.0054 | 0.0053 | 0.0003 | 0.0141 | -0.0827 | 0.0086 | 0.0160 | -0.0047 | 0.0042 | -0.0142 | 0.0131 |
| x_12 | -0.0863 | 0.0055 | 0.0210 | -0.0079 | 0.0008 | -0.0018 | 0.0159 | -0.0798 | 0.0033 | 0.0176 | -0.0084 | -0.0008 | -0.0056 | 0.0178 |
| x_13 | -0.0636 | -0.0051 | 0.0192 | -0.0140 | -0.0009 | 0.0355 | 0.0006 | -0.0683 | 0.0025 | 0.0169 | -0.0139 | -0.0022 | 0.0233 | -0.0019 |
$m_0$
| state | mean |
|---|---|
| x_0 | 0.0011 |
| x_1 | -0.0008 |
| x_2 | -0.0004 |
| x_3 | -0.0001 |
| x_4 | -0.0002 |
| x_5 | 0.0003 |
| x_6 | -0.0000 |
| x_7 | -0.0008 |
| x_8 | 0.0006 |
| x_9 | 0.0003 |
| x_10 | -0.0005 |
| x_11 | -0.0003 |
| x_12 | -0.0012 |
| x_13 | -0.0000 |
$P_0$
| state | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 2.9565 | -0.0008 | 0.0011 | 0.0001 | -0.0007 | -0.0009 | 0.0002 | -0.0001 | 0.0011 | -0.0006 | 0.0005 | -0.0011 | -0.0014 | 0.0004 |
| x_1 | -0.0008 | 2.9605 | -0.0000 | 0.0004 | 0.0015 | -0.0006 | 0.0000 | 0.0005 | -0.0009 | 0.0003 | -0.0002 | 0.0002 | 0.0015 | -0.0003 |
| x_2 | 0.0011 | -0.0000 | 2.9587 | -0.0004 | -0.0004 | 0.0008 | -0.0002 | 0.0003 | -0.0002 | -0.0003 | 0.0003 | 0.0003 | -0.0002 | -0.0002 |
| x_3 | 0.0001 | 0.0004 | -0.0004 | 2.9591 | 0.0011 | -0.0002 | -0.0003 | 0.0007 | -0.0005 | -0.0001 | -0.0006 | -0.0005 | 0.0007 | 0.0000 |
| x_4 | -0.0007 | 0.0015 | -0.0004 | 0.0011 | 2.9663 | -0.0018 | -0.0008 | -0.0026 | 0.0005 | 0.0010 | -0.0003 | 0.0005 | 0.0002 | -0.0005 |
| x_5 | -0.0009 | -0.0006 | 0.0008 | -0.0002 | -0.0018 | 2.9595 | 0.0005 | 0.0017 | 0.0001 | -0.0009 | 0.0002 | -0.0014 | -0.0003 | 0.0000 |
| x_6 | 0.0002 | 0.0000 | -0.0002 | -0.0003 | -0.0008 | 0.0005 | 2.9590 | 0.0010 | -0.0003 | -0.0002 | -0.0001 | -0.0002 | 0.0005 | 0.0001 |
| x_7 | -0.0001 | 0.0005 | 0.0003 | 0.0007 | -0.0026 | 0.0017 | 0.0010 | 2.9589 | -0.0008 | 0.0004 | -0.0001 | 0.0009 | 0.0007 | -0.0010 |
| x_8 | 0.0011 | -0.0009 | -0.0002 | -0.0005 | 0.0005 | 0.0001 | -0.0003 | -0.0008 | 2.9602 | -0.0002 | 0.0002 | -0.0001 | -0.0015 | 0.0003 |
| x_9 | -0.0006 | 0.0003 | -0.0003 | -0.0001 | 0.0010 | -0.0009 | -0.0002 | 0.0004 | -0.0002 | 2.9588 | -0.0002 | -0.0003 | 0.0005 | 0.0003 |
| x_10 | 0.0005 | -0.0002 | 0.0003 | -0.0006 | -0.0003 | 0.0002 | -0.0001 | -0.0001 | 0.0002 | -0.0002 | 2.9597 | 0.0003 | -0.0004 | 0.0001 |
| x_11 | -0.0011 | 0.0002 | 0.0003 | -0.0005 | 0.0005 | -0.0014 | -0.0002 | 0.0009 | -0.0001 | -0.0003 | 0.0003 | 2.9583 | 0.0005 | 0.0004 |
| x_12 | -0.0014 | 0.0015 | -0.0002 | 0.0007 | 0.0002 | -0.0003 | 0.0005 | 0.0007 | -0.0015 | 0.0005 | -0.0004 | 0.0005 | 2.9619 | -0.0007 |
| x_13 | 0.0004 | -0.0003 | -0.0002 | 0.0000 | -0.0005 | 0.0000 | 0.0001 | -0.0010 | 0.0003 | 0.0003 | 0.0001 | 0.0004 | -0.0007 | 2.9587 |
show_results(learn_TA, items = items_TA, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=4518, shift=15, var_sel=['TA'], gap_len=12), MeteoImpItem(i=3825, shift=5, var_sel=['TA'], gap_len=12), MeteoImpItem(i=3706, shift=-25, var_sel=['TA'], gap_len=12), MeteoImpItem(i=4397, shift=15, var_sel=['TA'], gap_len=12)]
TA - 96
after the first trainig fine tune the model for longer gaps
dls_TA96 = imp_dataloader(haiB, hai_eraB, var_sel = 'TA', block_len=300, gap_len=96, bs=20, control_lags=[1], n_rep=5).cpu()model_TA96 = model_TA.copy()save_models_TA96 = SaveModelsBatch(times_epoch=5)items_TA96 = random.choices(dls_TA96.valid.items, k=4)learn_TA96 = Learner(dls_TA96, model_TA96, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_TA96], metrics=rmse_gap)show_results(learn_TA96, items = items_TA96, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=723, shift=90, var_sel=['TA'], gap_len=96), MeteoImpItem(i=718, shift=30, var_sel=['TA'], gap_len=96), MeteoImpItem(i=706, shift=-150, var_sel=['TA'], gap_len=96), MeteoImpItem(i=756, shift=90, var_sel=['TA'], gap_len=96)]
learn_TA96.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | -61.745678 | -63.454943 | 0.116816 | 22:50 |
learn_TA96.recorder.plot_loss()
show_results(learn_TA96, items = items_TA96, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=723, shift=90, var_sel=['TA'], gap_len=96), MeteoImpItem(i=718, shift=30, var_sel=['TA'], gap_len=96), MeteoImpItem(i=706, shift=-150, var_sel=['TA'], gap_len=96), MeteoImpItem(i=756, shift=90, var_sel=['TA'], gap_len=96)]
torch.save(learn_TA96.model, "trained_4_feb_TA_gap_96_v1.pickle")All variables - 12 varying
Model Av (All varying) all variables with a varing numbers of variables
dls_Av = imp_dataloader(haiB, hai_eraB, var_sel = gen_var_sel(list(haiB.columns)), block_len=100, gap_len=12, bs=20, control_lags=[1], n_rep=10).cpu()model_Av = model_TA.copy() save_models_Av = SaveModelsBatch(times_epoch=1) # save once per repetitionitems_Av = random.choices(dls_Av.valid.items, k=4)learn_Av = Learner(dls_Av, model_Av, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_Av], metrics=rmse_gap)learn_Av.fit(2, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 27.495375 | 25.322661 | 0.489185 | 26:10 |
| 1 | 14.247212 | 13.797795 | 0.408078 | 26:31 |
learn_Av.recorder.plot_loss()
with with_settings(learn_Av.model, use_conditional =False): #speed up training
learn_Av.fit(2, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 7.901138 | 7.892225 | 0.360798 | 23:14 |
| 1 | 2.295464 | 4.088155 | 0.336661 | 23:39 |
learn_Av.recorder.plot_loss()
torch.save(learn_Av.model, "trained_4_feb_All_gap_varying_12_v1.pickle")show_results(learn_Av, items=items_Av, control_map=control_map)[MeteoImpItem(i=1867, shift=0, var_sel=['TA'], gap_len=12), MeteoImpItem(i=2001, shift=-50, var_sel=['PA', 'VPD', 'LW_IN', 'TA', 'WS'], gap_len=12), MeteoImpItem(i=1960, shift=-40, var_sel=['TA', 'PA', 'P', 'SW_IN', 'VPD', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=2163, shift=30, var_sel=['TA'], gap_len=12)]
with with_settings(learn_Av.model, use_conditional =False): #speed up training
learn_Av.fit(2, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 3.646232 | 1.610648 | 0.321632 | 24:10 |
| 1 | -1.735119 | -0.024182 | 0.312380 | 24:34 |
learn_Av.recorder.plot_loss()
torch.save(learn_Av.model, "trained_4_feb_All_gap_varying_12_v2.pickle")show_results(learn_Av, items=items_Av, control_map=control_map)[MeteoImpItem(i=1867, shift=0, var_sel=['TA'], gap_len=12), MeteoImpItem(i=2001, shift=-50, var_sel=['PA', 'VPD', 'LW_IN', 'TA', 'WS'], gap_len=12), MeteoImpItem(i=1960, shift=-40, var_sel=['TA', 'PA', 'P', 'SW_IN', 'VPD', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=2163, shift=30, var_sel=['TA'], gap_len=12)]
All variables - 12 all gap
dls_Aa = imp_dataloader(haiB, hai_eraB, var_sel = list(haiB.columns), block_len=70, gap_len=12, bs=20, control_lags=[1], n_rep=10).cpu()model_Aa = model_Av.copy()
model_Aa.use_conditional = Falsesave_models_Aa = SaveModelsBatch(times_epoch=1) # save once per repetitionitems_Aa = random.choices(dls_Aa.valid.items, k=4)learn_Aa = Learner(dls_Aa, model_Aa, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_Aa], metrics=rmse_gap)learn_Aa.fit(2, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | -3.860496 | -2.163738 | 0.297889 | 32:06 |
| 1 | -3.991504 | -3.813984 | 0.290560 | 32:34 |
learn_Aa.recorder.plot_loss()
torch.save(learn_Aa.model, "trained_4_feb_All_gap_all_12_v1.pickle")show_results(learn_Aa, items=items_Aa, control_map=control_map)[MeteoImpItem(i=3064, shift=7, var_sel=['P', 'VPD', 'WS', 'SW_IN', 'PA', 'TA'], gap_len=12), MeteoImpItem(i=2776, shift=7, var_sel=['VPD', 'PA', 'SW_IN', 'WS'], gap_len=12), MeteoImpItem(i=3253, shift=-21, var_sel=['VPD', 'SW_IN'], gap_len=12), MeteoImpItem(i=2716, shift=14, var_sel=['SW_IN'], gap_len=12)]
SW_IN - gap 12
dls_SW_IN = imp_dataloader(haiB, hai_eraB, var_sel = 'SW_IN', block_len=50, gap_len=12, bs=20, control_lags=[1], n_rep=5).cpu()model_SW_IN = model_Av.copy()save_models_SW_IN = SaveModelsBatch(times_epoch=10)items_SW_IN = random.choices(dls_SW_IN.valid.items, k=4)learn_SW_IN = Learner(dls_SW_IN, model_SW_IN, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_SW_IN], metrics=rmse_gap)learn_SW_IN.fit(3, 5e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 1.571930 | 2.161097 | 0.229693 | 12:20 |
| 1 | 1.701283 | 1.642652 | 0.216893 | 12:26 |
| 2 | 1.033681 | 1.752522 | 0.214100 | 12:50 |
learn_SW_IN.recorder.plot_loss()
torch.save(learn_SW_IN.model, "trained_4_feb_SW_IN_gap_12_v1.pickle")learn_SW_INKalman Filter (7 obs, 14 state, 14 contr)
$A$
| state | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 1.2092 | 0.0099 | -0.1905 | 0.0169 | -0.0984 | -0.0086 | -0.0375 | 0.8756 | 0.0942 | 0.2282 | -0.1515 | -0.0565 | -0.0419 | -0.0731 |
| x_1 | 0.0250 | 1.1628 | -0.1911 | 0.1139 | -0.0933 | -0.0061 | -0.0202 | 0.0043 | 0.8123 | 0.1668 | -0.1492 | 0.0625 | 0.0224 | 0.0082 |
| x_2 | 0.0058 | -0.0328 | 1.2095 | -0.1337 | 0.0424 | 0.0783 | 0.0067 | -0.0326 | -0.0048 | 0.6848 | 0.1042 | -0.0030 | -0.0034 | 0.0172 |
| x_3 | 0.0506 | 0.0430 | -0.1128 | 1.1909 | -0.0675 | -0.0662 | 0.0139 | 0.0205 | 0.0210 | 0.1381 | 0.6624 | 0.0020 | 0.0031 | -0.0198 |
| x_4 | -0.1150 | -0.0565 | 0.1214 | -0.1329 | 1.1548 | 0.0006 | 0.0438 | 0.0248 | 0.0160 | -0.1514 | 0.2205 | 0.7190 | -0.0158 | -0.0161 |
| x_5 | -0.0051 | -0.0034 | 0.1459 | -0.1069 | 0.0482 | 1.0111 | 0.0273 | 0.1421 | -0.0900 | -0.1165 | 0.1743 | -0.0579 | 0.6217 | 0.0555 |
| x_6 | -0.1867 | 0.0374 | 0.0283 | 0.0998 | 0.1161 | -0.1061 | 1.0440 | 0.0212 | -0.1477 | 0.0734 | -0.0844 | 0.0142 | 0.1911 | 1.1859 |
| x_7 | 0.1577 | -0.0190 | -0.0549 | 0.0844 | 0.0527 | 0.0865 | 0.0427 | 0.9606 | -0.0319 | -0.1238 | 0.0597 | 0.0510 | 0.0823 | 0.0125 |
| x_8 | 0.0562 | 0.1505 | -0.0321 | 0.0300 | -0.0300 | 0.0235 | -0.0623 | -0.0047 | 0.9781 | -0.0897 | 0.0459 | 0.0068 | 0.0778 | 0.0129 |
| x_9 | 0.0766 | -0.0076 | 0.1531 | -0.0059 | -0.0445 | -0.0611 | 0.0210 | -0.0223 | -0.0109 | 1.1227 | -0.0934 | 0.0140 | -0.0822 | -0.0101 |
| x_10 | -0.0910 | -0.0241 | -0.0252 | 0.1167 | 0.0297 | 0.0981 | -0.0182 | -0.0112 | 0.0018 | -0.1045 | 1.1444 | 0.0188 | 0.0485 | 0.0639 |
| x_11 | 0.0049 | -0.0412 | 0.0115 | -0.0502 | 0.1451 | -0.0527 | -0.0148 | 0.0156 | -0.0171 | 0.0801 | -0.0582 | 1.0207 | -0.0824 | 0.0098 |
| x_12 | -0.1013 | 0.0220 | -0.0262 | -0.0522 | 0.0147 | 0.1882 | 0.0488 | -0.0134 | 0.0225 | 0.0985 | -0.0930 | -0.0503 | 0.9256 | -0.0362 |
| x_13 | -0.0310 | 0.0349 | 0.0375 | -0.0787 | 0.0667 | -0.0509 | 0.1694 | -0.0421 | 0.0158 | -0.0043 | 0.0215 | 0.0122 | -0.1242 | 0.8931 |
$Q$
| state | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 0.0535 | -0.0058 | 0.0094 | -0.0022 | 0.0197 | -0.0093 | 0.0291 | 0.0008 | -0.0093 | 0.0033 | 0.0175 | -0.0064 | 0.0123 | -0.0169 |
| x_1 | -0.0058 | 0.0853 | 0.0153 | -0.0184 | 0.0206 | -0.0001 | -0.0288 | 0.0110 | 0.0101 | 0.0152 | -0.0061 | -0.0041 | 0.0050 | 0.0180 |
| x_2 | 0.0094 | 0.0153 | 0.0469 | 0.0356 | -0.0255 | -0.0048 | -0.0373 | -0.0236 | -0.0104 | 0.0321 | 0.0009 | 0.0050 | 0.0260 | 0.0117 |
| x_3 | -0.0022 | -0.0184 | 0.0356 | 0.0919 | 0.0083 | -0.0059 | -0.0550 | -0.0075 | -0.0501 | 0.0344 | 0.0013 | 0.0492 | 0.0305 | 0.0328 |
| x_4 | 0.0197 | 0.0206 | -0.0255 | 0.0083 | 0.1291 | -0.0189 | 0.0220 | 0.0355 | -0.0023 | -0.0047 | -0.0040 | 0.0122 | -0.0185 | -0.0110 |
| x_5 | -0.0093 | -0.0001 | -0.0048 | -0.0059 | -0.0189 | 0.0369 | -0.0015 | -0.0240 | 0.0130 | -0.0187 | -0.0059 | -0.0153 | 0.0204 | -0.0223 |
| x_6 | 0.0291 | -0.0288 | -0.0373 | -0.0550 | 0.0220 | -0.0015 | 0.2657 | 0.0163 | 0.0369 | -0.0117 | 0.0199 | -0.0298 | -0.0415 | -0.0983 |
| x_7 | 0.0008 | 0.0110 | -0.0236 | -0.0075 | 0.0355 | -0.0240 | 0.0163 | 0.0855 | -0.0254 | 0.0138 | -0.0050 | 0.0298 | -0.0230 | 0.0170 |
| x_8 | -0.0093 | 0.0101 | -0.0104 | -0.0501 | -0.0023 | 0.0130 | 0.0369 | -0.0254 | 0.1099 | -0.0127 | -0.0089 | -0.0276 | -0.0180 | -0.0152 |
| x_9 | 0.0033 | 0.0152 | 0.0321 | 0.0344 | -0.0047 | -0.0187 | -0.0117 | 0.0138 | -0.0127 | 0.0855 | 0.0023 | 0.0067 | 0.0010 | -0.0180 |
| x_10 | 0.0175 | -0.0061 | 0.0009 | 0.0013 | -0.0040 | -0.0059 | 0.0199 | -0.0050 | -0.0089 | 0.0023 | 0.0590 | 0.0247 | -0.0185 | 0.0288 |
| x_11 | -0.0064 | -0.0041 | 0.0050 | 0.0492 | 0.0122 | -0.0153 | -0.0298 | 0.0298 | -0.0276 | 0.0067 | 0.0247 | 0.1196 | -0.0086 | 0.0207 |
| x_12 | 0.0123 | 0.0050 | 0.0260 | 0.0305 | -0.0185 | 0.0204 | -0.0415 | -0.0230 | -0.0180 | 0.0010 | -0.0185 | -0.0086 | 0.0850 | 0.0112 |
| x_13 | -0.0169 | 0.0180 | 0.0117 | 0.0328 | -0.0110 | -0.0223 | -0.0983 | 0.0170 | -0.0152 | -0.0180 | 0.0288 | 0.0207 | 0.0112 | 0.3036 |
$b$
| state | offset |
|---|---|
| x_0 | -0.0035 |
| x_1 | -0.0264 |
| x_2 | 0.0016 |
| x_3 | -0.0055 |
| x_4 | -0.0095 |
| x_5 | -0.0025 |
| x_6 | -0.0167 |
| x_7 | 0.0090 |
| x_8 | 0.0001 |
| x_9 | 0.0003 |
| x_10 | 0.0088 |
| x_11 | -0.0035 |
| x_12 | 0.0114 |
| x_13 | -0.0019 |
$H$
| variable | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| y_0 | 0.1119 | -0.1229 | 0.5709 | -0.2683 | 0.0774 | -0.0004 | 0.0066 | 0.0550 | -0.0632 | 0.0880 | -0.0444 | 0.0467 | -0.0558 | 0.0246 |
| y_1 | 0.7579 | 0.1629 | 0.2327 | -0.0197 | -0.0384 | -0.2112 | -0.0065 | 0.1209 | -0.0729 | 0.2284 | -0.0980 | 0.0492 | 0.1636 | -0.0476 |
| y_2 | 0.0352 | 0.0115 | 0.5271 | 0.4768 | -0.0234 | 0.0289 | -0.0082 | 0.0849 | 0.0510 | -0.0186 | 0.0404 | -0.0603 | -0.1070 | -0.0790 |
| y_3 | 0.0269 | 0.0600 | 0.1747 | -0.1470 | -0.0931 | -0.6539 | 0.0019 | -0.0266 | 0.0604 | -0.0557 | -0.0043 | -0.0811 | -0.0164 | -0.0517 |
| y_4 | 0.0068 | -0.1011 | -0.0548 | -0.1172 | -0.0165 | 0.0706 | 1.3534 | -0.0528 | 0.0761 | -0.1036 | 0.1065 | -0.1210 | 0.0232 | -0.3115 |
| y_5 | 0.0833 | -0.0442 | -0.1637 | 0.0456 | 0.9350 | 0.1287 | -0.0171 | -0.0895 | -0.0563 | -0.1668 | 0.1400 | 0.1260 | -0.0767 | 0.0628 |
| y_6 | -0.1897 | -0.8226 | -0.0693 | 0.0926 | 0.0038 | -0.1236 | 0.0132 | -0.0346 | -0.1286 | 0.1014 | -0.0264 | 0.0526 | -0.0744 | 0.0379 |
$R$
| variable | y_0 | y_1 | y_2 | y_3 | y_4 | y_5 | y_6 |
|---|---|---|---|---|---|---|---|
| y_0 | 0.0014 | -0.0038 | 0.0011 | 0.0035 | 0.0017 | 0.0021 | -0.0001 |
| y_1 | -0.0038 | 0.0176 | -0.0059 | -0.0111 | 0.0069 | 0.0018 | 0.0015 |
| y_2 | 0.0011 | -0.0059 | 0.0058 | 0.0040 | 0.0065 | -0.0014 | -0.0039 |
| y_3 | 0.0035 | -0.0111 | 0.0040 | 0.0107 | 0.0027 | 0.0077 | -0.0039 |
| y_4 | 0.0017 | 0.0069 | 0.0065 | 0.0027 | 0.0722 | -0.0017 | -0.0053 |
| y_5 | 0.0021 | 0.0018 | -0.0014 | 0.0077 | -0.0017 | 0.0376 | -0.0054 |
| y_6 | -0.0001 | 0.0015 | -0.0039 | -0.0039 | -0.0053 | -0.0054 | 0.0136 |
$d$
| variable | offset |
|---|---|
| y_0 | -0.0007 |
| y_1 | -0.0063 |
| y_2 | -0.0204 |
| y_3 | 0.0145 |
| y_4 | -0.0090 |
| y_5 | -0.0468 |
| y_6 | -0.0133 |
$B$
| state | c_0 | c_1 | c_2 | c_3 | c_4 | c_5 | c_6 | c_7 | c_8 | c_9 | c_10 | c_11 | c_12 | c_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 0.1186 | -0.8485 | -0.0630 | 0.1522 | 0.0082 | -0.0564 | -0.0824 | 0.1909 | 0.8190 | -0.0354 | 0.1066 | -0.0523 | 0.0139 | -0.0462 |
| x_1 | 0.2492 | 0.0849 | -0.0074 | 0.0056 | -0.0049 | -0.0478 | 0.8928 | 0.1398 | 0.0167 | -0.0199 | -0.0437 | 0.0060 | 0.0587 | -0.7312 |
| x_2 | -0.8873 | -0.0521 | -0.4452 | 0.0263 | 0.0363 | 0.1346 | 0.1307 | 0.6552 | 0.0144 | 0.4837 | 0.1016 | 0.0027 | -0.0666 | -0.1407 |
| x_3 | 0.5261 | -0.0119 | -0.8469 | -0.0277 | -0.0336 | -0.1063 | -0.0656 | -0.2830 | -0.1705 | 0.8257 | -0.0865 | 0.0056 | 0.0958 | 0.0383 |
| x_4 | -0.1738 | 0.0272 | 0.0501 | 0.1181 | -0.0083 | -0.9154 | 0.0459 | -0.1750 | 0.1252 | 0.0355 | -0.1135 | 0.0329 | 0.8490 | -0.0108 |
| x_5 | -0.0502 | -0.1978 | 0.0494 | 0.9221 | -0.0033 | 0.2204 | 0.0062 | -0.1387 | -0.0034 | -0.0207 | -0.9759 | 0.0176 | -0.1051 | -0.0576 |
| x_6 | -0.0261 | 0.0876 | -0.0362 | -0.0741 | -0.6339 | 0.0092 | 0.0529 | -0.0338 | 0.0857 | -0.0185 | -0.0145 | 0.6687 | -0.0498 | 0.0243 |
| x_7 | 0.0388 | -0.1206 | -0.0035 | 0.1071 | -0.0304 | -0.0273 | -0.0187 | 0.0207 | -0.1097 | 0.0004 | 0.1018 | -0.0148 | -0.0261 | -0.0148 |
| x_8 | 0.0774 | -0.0308 | -0.0022 | 0.0180 | 0.0222 | 0.0056 | 0.0632 | 0.0798 | -0.0283 | -0.0045 | 0.0180 | -0.0022 | 0.0181 | 0.0522 |
| x_9 | -0.0570 | -0.0036 | -0.0621 | -0.0331 | 0.0118 | 0.0193 | 0.0259 | -0.0540 | -0.0366 | -0.0568 | -0.0314 | 0.0155 | 0.0005 | 0.0092 |
| x_10 | 0.0497 | 0.0343 | -0.0569 | 0.0741 | 0.0015 | -0.0078 | -0.0098 | 0.0436 | 0.0464 | -0.0525 | 0.0730 | -0.0180 | -0.0003 | 0.0007 |
| x_11 | -0.0501 | 0.0310 | 0.0116 | -0.0328 | 0.0160 | -0.0563 | -0.0090 | -0.0572 | 0.0090 | 0.0115 | -0.0320 | 0.0180 | -0.0631 | -0.0271 |
| x_12 | -0.0648 | 0.0881 | 0.0226 | 0.0939 | -0.0059 | -0.0011 | 0.0477 | -0.0567 | 0.0641 | 0.0235 | 0.0896 | -0.0028 | -0.0068 | 0.0513 |
| x_13 | -0.0057 | 0.0274 | -0.0023 | -0.0298 | -0.1657 | 0.0082 | 0.0236 | -0.0124 | 0.0238 | 0.0026 | -0.0266 | 0.0452 | -0.0011 | 0.0150 |
$m_0$
| state | mean |
|---|---|
| x_0 | 0.0011 |
| x_1 | -0.0008 |
| x_2 | -0.0004 |
| x_3 | -0.0001 |
| x_4 | -0.0002 |
| x_5 | 0.0003 |
| x_6 | -0.0000 |
| x_7 | -0.0008 |
| x_8 | 0.0006 |
| x_9 | 0.0003 |
| x_10 | -0.0005 |
| x_11 | -0.0003 |
| x_12 | -0.0012 |
| x_13 | -0.0000 |
$P_0$
| state | x_0 | x_1 | x_2 | x_3 | x_4 | x_5 | x_6 | x_7 | x_8 | x_9 | x_10 | x_11 | x_12 | x_13 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| x_0 | 2.8754 | -0.0007 | 0.0011 | 0.0001 | -0.0005 | -0.0010 | 0.0001 | -0.0001 | 0.0010 | -0.0006 | 0.0007 | -0.0014 | -0.0013 | 0.0003 |
| x_1 | -0.0007 | 2.8798 | 0.0000 | 0.0004 | 0.0015 | -0.0006 | -0.0000 | 0.0004 | -0.0009 | 0.0003 | -0.0001 | 0.0001 | 0.0014 | -0.0003 |
| x_2 | 0.0011 | 0.0000 | 2.8779 | -0.0005 | -0.0004 | 0.0008 | -0.0002 | 0.0004 | -0.0002 | -0.0003 | 0.0003 | 0.0003 | -0.0002 | -0.0001 |
| x_3 | 0.0001 | 0.0004 | -0.0005 | 2.8785 | 0.0010 | -0.0002 | -0.0002 | 0.0008 | -0.0004 | -0.0001 | -0.0009 | -0.0003 | 0.0007 | 0.0001 |
| x_4 | -0.0005 | 0.0015 | -0.0004 | 0.0010 | 2.8854 | -0.0017 | -0.0008 | -0.0026 | 0.0005 | 0.0011 | -0.0001 | 0.0004 | 0.0002 | -0.0005 |
| x_5 | -0.0010 | -0.0006 | 0.0008 | -0.0002 | -0.0017 | 2.8786 | 0.0005 | 0.0016 | 0.0001 | -0.0009 | 0.0002 | -0.0014 | -0.0003 | 0.0000 |
| x_6 | 0.0001 | -0.0000 | -0.0002 | -0.0002 | -0.0008 | 0.0005 | 2.8783 | 0.0010 | -0.0002 | -0.0002 | -0.0002 | -0.0002 | 0.0005 | 0.0001 |
| x_7 | -0.0001 | 0.0004 | 0.0004 | 0.0008 | -0.0026 | 0.0016 | 0.0010 | 2.8781 | -0.0007 | 0.0004 | -0.0001 | 0.0009 | 0.0007 | -0.0009 |
| x_8 | 0.0010 | -0.0009 | -0.0002 | -0.0004 | 0.0005 | 0.0001 | -0.0002 | -0.0007 | 2.8794 | -0.0002 | 0.0001 | 0.0000 | -0.0015 | 0.0003 |
| x_9 | -0.0006 | 0.0003 | -0.0003 | -0.0001 | 0.0011 | -0.0009 | -0.0002 | 0.0004 | -0.0002 | 2.8781 | -0.0001 | -0.0003 | 0.0005 | 0.0003 |
| x_10 | 0.0007 | -0.0001 | 0.0003 | -0.0009 | -0.0001 | 0.0002 | -0.0002 | -0.0001 | 0.0001 | -0.0001 | 2.8795 | 0.0001 | -0.0004 | -0.0000 |
| x_11 | -0.0014 | 0.0001 | 0.0003 | -0.0003 | 0.0004 | -0.0014 | -0.0002 | 0.0009 | 0.0000 | -0.0003 | 0.0001 | 2.8775 | 0.0004 | 0.0004 |
| x_12 | -0.0013 | 0.0014 | -0.0002 | 0.0007 | 0.0002 | -0.0003 | 0.0005 | 0.0007 | -0.0015 | 0.0005 | -0.0004 | 0.0004 | 2.8811 | -0.0007 |
| x_13 | 0.0003 | -0.0003 | -0.0001 | 0.0001 | -0.0005 | 0.0000 | 0.0001 | -0.0009 | 0.0003 | 0.0003 | -0.0000 | 0.0004 | -0.0007 | 2.8780 |
show_results(learn_SW_IN, items = items_SW_IN, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=3927, shift=-15, var_sel=['SW_IN'], gap_len=12), MeteoImpItem(i=4045, shift=-5, var_sel=['SW_IN'], gap_len=12), MeteoImpItem(i=4390, shift=-25, var_sel=['SW_IN'], gap_len=12), MeteoImpItem(i=3968, shift=-25, var_sel=['SW_IN'], gap_len=12)]
SW_IN - 96
after the first trainig fine tune the model for longer gaps
dls_SW_IN96 = imp_dataloader(haiB, hai_eraB, var_sel = 'SW_IN', block_len=300, gap_len=96, bs=20, control_lags=[1], n_rep=5).cpu()model_SW_IN96 = model_SW_IN.copy()save_models_SW_IN96 = SaveModelsBatch(times_epoch=5)items_SW_IN96 = random.choices(dls_SW_IN96.valid.items, k=4)learn_SW_IN96 = Learner(dls_SW_IN96, model_SW_IN96, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_SW_IN96], metrics=rmse_gap)show_results(learn_SW_IN96, items = items_SW_IN96, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=660, shift=30, var_sel=['SW_IN'], gap_len=96), MeteoImpItem(i=684, shift=30, var_sel=['SW_IN'], gap_len=96), MeteoImpItem(i=684, shift=90, var_sel=['SW_IN'], gap_len=96), MeteoImpItem(i=632, shift=-90, var_sel=['SW_IN'], gap_len=96)]
learn_SW_IN96.fit(2, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 23.111818 | 21.649970 | 0.277776 | 22:20 |
| 1 | 23.287213 | 21.537204 | 0.277468 | 22:06 |
learn_SW_IN96.recorder.plot_loss()
show_results(learn_SW_IN96, items = items_SW_IN96, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=660, shift=30, var_sel=['SW_IN'], gap_len=96), MeteoImpItem(i=684, shift=30, var_sel=['SW_IN'], gap_len=96), MeteoImpItem(i=684, shift=90, var_sel=['SW_IN'], gap_len=96), MeteoImpItem(i=632, shift=-90, var_sel=['SW_IN'], gap_len=96)]
torch.save(learn_SW_IN96.model, "trained_4_feb_SW_IN_gap_96_v1.pickle")Control
No Control - TA 12
dls_TA_nc = imp_dataloader(haiB, hai_eraB, var_sel = 'TA', block_len=50, gap_len=12, bs=20, control_lags=[1], n_rep=1).cpu()model_TA_nc = model_TA96.copy()
model_TA_nc.use_control = Falsesave_models_TA_nc = SaveModelsBatch(times_epoch=2)items_TA_nc = random.choices(dls_TA_nc.valid.items, k=4)learn_TA_nc = Learner(dls_TA_nc, model_TA_nc, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_TA_nc], metrics=rmse_gap)learn_TA_nc.fit(1, 5e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | -18.464700 | -18.834046 | 0.040957 | 02:33 |
learn_TA_nc.recorder.plot_loss()
torch.save(learn_TA_nc.model, "trained_4_feb_TA_gap_12_no_control_v1.pickle")show_results(learn_TA_nc, items = items_TA_nc, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=3919, shift=15, var_sel=['TA'], gap_len=12), MeteoImpItem(i=4068, shift=15, var_sel=['TA'], gap_len=12), MeteoImpItem(i=4114, shift=-15, var_sel=['TA'], gap_len=12), MeteoImpItem(i=4180, shift=-5, var_sel=['TA'], gap_len=12)]
No Control - TA 96
dls_TA96_nc = imp_dataloader(haiB, hai_eraB, var_sel = 'TA', block_len=300, gap_len=96, bs=20, control_lags=[1], n_rep=5).cpu()model_TA96_nc = model_TA_nc.copy()
model_TA96_nc.use_control = False/home/simone/anaconda3/envs/data-science/lib/python3.10/site-packages/fastai/callback/core.py:69: UserWarning: You are shadowing an attribute (__class__) that exists in the learner. Use `self.learn.__class__` to avoid this
warn(f"You are shadowing an attribute ({name}) that exists in the learner. Use `self.learn.{name}` to avoid this")
save_models_TA96_nc = SaveModelsBatch(times_epoch=2)items_TA96_nc = random.choices(dls_TA96_nc.valid.items, k=4)learn_TA96_nc = Learner(dls_TA96_nc, model_TA96_nc, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_TA96_nc], metrics=rmse_gap)show_results(learn_TA96_nc, items = items_TA96_nc, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=757, shift=-90, var_sel=['TA'], gap_len=96), MeteoImpItem(i=711, shift=-90, var_sel=['TA'], gap_len=96), MeteoImpItem(i=743, shift=-150, var_sel=['TA'], gap_len=96), MeteoImpItem(i=625, shift=-30, var_sel=['TA'], gap_len=96)]
learn_TA96_nc.fit(2, 5e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | -16.875112 | -38.792134 | 0.158970 | 24:20 |
| 1 | -36.028650 | -42.915542 | 0.153475 | 22:35 |
learn_TA96_nc.recorder.plot_loss()
torch.save(learn_TA96_nc.model, "trained_4_feb_TA_gap_96_no_control_v1.pickle")show_results(learn_TA96_nc, items = items_TA96_nc, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=757, shift=-90, var_sel=['TA'], gap_len=96), MeteoImpItem(i=711, shift=-90, var_sel=['TA'], gap_len=96), MeteoImpItem(i=743, shift=-150, var_sel=['TA'], gap_len=96), MeteoImpItem(i=625, shift=-30, var_sel=['TA'], gap_len=96)]
No Control - All variables - 12 all gap
dls_Aa96_nc = imp_dataloader(haiB, hai_eraB, var_sel = list(haiB.columns), block_len=70, gap_len=12, bs=20, control_lags=[1], n_rep=2).cpu()model_Aa96_nc = model_Aa96.copy()
model_Aa96_nc.use_control = Falsesave_models_Aa96_nc= SaveModelsBatch(times_epoch=1) # save once per repetitionitems_Aa96_nc = random.choices(dls_Aa96_nc.valid.items, k=4)learn_Aa96_nc = Learner(dls_Aa96_nc, model_Aa96_nc, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_Aa96_nc], metrics=rmse_gap)show_results(learn_Aa96_nc, items=items_Aa96_nc, control_map=control_map)[MeteoImpItem(i=3064, shift=7, var_sel=['P', 'VPD', 'WS', 'SW_IN', 'PA', 'TA'], gap_len=12), MeteoImpItem(i=2776, shift=7, var_sel=['VPD', 'PA', 'SW_IN', 'WS'], gap_len=12), MeteoImpItem(i=3253, shift=-21, var_sel=['VPD', 'SW_IN'], gap_len=12), MeteoImpItem(i=2716, shift=14, var_sel=['SW_IN'], gap_len=12)]
learn_Aa96_nc.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 38.744036 | 30.468859 | 0.436350 | 03:39 |
learn_Aa96_nc.recorder.plot_loss()
learn_Aa96_nc.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 12.341979 | 12.002963 | 0.392356 | 03:51 |
learn_Aa96_nc.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 5.277773 | 4.664958 | 0.367391 | 37:31 |
learn_Aa96_nc.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 1.912588 | 0.122255 | 0.352851 | 04:10 |
learn_Aa96_nc.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | -2.542222 | -3.259210 | 0.344657 | 03:51 |
learn_Aa96_nc.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | -5.897630 | -5.467389 | 0.339895 | 04:01 |
learn_Aa96_nc.recorder.plot_loss()
torch.save(learn_Aa96_nc.model, "trained_4_feb_All_gap_all_12_no_control_v1.pickle")show_results(learn_Aa96_nc, items=items_Aa96_nc, control_map=control_map)[MeteoImpItem(i=3077, shift=-35, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=3195, shift=-35, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=3230, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=2910, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12)]
No Control - All variables - 24 all gap
dls_Aa24_nc = imp_dataloader(haiB, hai_eraB, var_sel = list(haiB.columns), block_len=150, gap_len=24, bs=20, control_lags=[1], n_rep=2).cpu()model_Aa24_nc = model_Aa_nc.copy()
model_Aa24_nc.use_control = Falsesave_models_Aa24_nc= SaveModelsBatch(times_epoch=1) # save once per repetitionitems_Aa24_nc = random.choices(dls_Aa24_nc.valid.items, k=4)learn_Aa24_nc = Learner(dls_Aa24_nc, model_Aa24_nc, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_Aa24_nc], metrics=rmse_gap)show_results(learn_Aa24_nc, items=items_Aa24_nc, control_map=control_map)[MeteoImpItem(i=1439, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24), MeteoImpItem(i=1222, shift=-75, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24), MeteoImpItem(i=1376, shift=-75, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24), MeteoImpItem(i=1364, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24)]
learn_Aa24_nc.fit(5, 3e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 41.673347 | 42.895707 | 0.457301 | 04:47 |
| 1 | 42.247850 | 40.339583 | 0.451268 | 04:37 |
| 2 | 34.196943 | 38.366525 | 0.445817 | 04:47 |
| 3 | 32.270157 | 36.991478 | 0.443363 | 04:42 |
| 4 | 33.566557 | 35.243634 | 0.436804 | 04:54 |
learn_Aa24_nc.recorder.plot_loss()
torch.save(learn_Aa24_nc.model, "trained_4_feb_All_gap_all_24_no_control_v1.pickle")show_results(learn_Aa24_nc, items=items_Aa24_nc, control_map=control_map)[MeteoImpItem(i=1439, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24), MeteoImpItem(i=1222, shift=-75, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24), MeteoImpItem(i=1376, shift=-75, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24), MeteoImpItem(i=1364, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=24)]
No Control - All variables - 24 varying
dls_Av24_nc = imp_dataloader(haiB, hai_eraB, var_sel = gen_var_sel(list(haiB.columns)), block_len=150, gap_len=24, bs=20, control_lags=[1], n_rep=2).cpu()model_Av24_nc = model_Aa24_nc.copy()
model_Av24_nc.use_control = Falsesave_models_Av24_nc= SaveModelsBatch(times_epoch=1) # save once per repetitionitems_Av24_nc = random.choices(dls_Av24_nc.valid.items, k=4)learn_Av24_nc = Learner(dls_Av24_nc, model_Av24_nc, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_Av24_nc], metrics=rmse_gap)show_results(learn_Av24_nc, items=items_Av24_nc, control_map=control_map)[MeteoImpItem(i=1438, shift=0, var_sel=['SW_IN', 'TA', 'PA', 'LW_IN', 'VPD', 'WS', 'P'], gap_len=24), MeteoImpItem(i=1275, shift=0, var_sel=['WS', 'SW_IN', 'TA', 'LW_IN'], gap_len=24), MeteoImpItem(i=1316, shift=-75, var_sel=['VPD', 'WS', 'TA', 'P', 'LW_IN', 'PA', 'SW_IN'], gap_len=24), MeteoImpItem(i=1273, shift=-75, var_sel=['P', 'TA', 'SW_IN', 'LW_IN', 'WS'], gap_len=24)]
learn_Av24_nc.fit(2, 3e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 18.450633 | 16.309334 | 0.376543 | 06:58 |
| 1 | 15.670756 | 16.164595 | 0.374336 | 06:24 |
learn_Av24_nc.recorder.plot_loss()
torch.save(learn_Av24_nc.model, "trained_4_feb_All_gap_varying_24_no_control_v1.pickle")show_results(learn_Av24_nc, items=items_Av24_nc, control_map=control_map)No Control - All variables - 36 all gap
dls_Aa36_nc = imp_dataloader(haiB, hai_eraB, var_sel = list(haiB.columns), block_len=150, gap_len=36, bs=20, control_lags=[1], n_rep=2).cpu()model_Aa36_nc = model_Aa24_nc.copy()
model_Aa36_nc.use_control = Falsesave_models_Aa36_nc= SaveModelsBatch(times_epoch=1) # save once per repetitionitems_Aa36_nc = random.choices(dls_Aa36_nc.valid.items, k=4)learn_Aa36_nc = Learner(dls_Aa36_nc, model_Aa36_nc, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_Aa36_nc], metrics=rmse_gap)show_results(learn_Aa36_nc, items=items_Aa36_nc, control_map=control_map)[MeteoImpItem(i=1384, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=36), MeteoImpItem(i=1362, shift=-75, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=36), MeteoImpItem(i=1410, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=36), MeteoImpItem(i=1505, shift=-75, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=36)]
learn_Aa36_nc.fit(2, 3e-4)
0.00% [0/2 00:00<?]
| epoch | train_loss | valid_loss | rmse_gap | time |
|---|
0.00% [0/121 00:00<?]
ValueError: Expected parameter covariance_matrix (Tensor of shape (7, 7)) of distribution MultivariateNormal(loc: torch.Size([7]), covariance_matrix: torch.Size([7, 7])) to satisfy the constraint PositiveDefinite(), but found invalid values:
tensor([[ 1.6642e-02, 4.3281e-03, 9.5652e-03, 2.6394e-03, 2.6164e-02,
1.5804e-02, 3.9479e-02],
[ 4.3281e-03, 5.1357e-02, 3.1779e-03, 4.4854e-03, -1.5141e-02,
3.6567e-03, -3.5843e-02],
[ 9.5652e-03, 3.1779e-03, 1.6632e-02, -4.0737e-04, 1.4566e-02,
1.5618e-02, 1.2989e-02],
[ 2.6394e-03, 4.4854e-03, -4.0737e-04, 4.9380e-03, -9.9333e-03,
-1.2459e-03, -7.2656e-03],
[ 2.6164e-02, -1.5141e-02, 1.4566e-02, -9.9333e-03, 7.9823e-01,
2.5349e-02, 5.7989e-02],
[ 1.5804e-02, 3.6567e-03, 1.5618e-02, -1.2459e-03, 2.5349e-02,
1.0991e-01, 6.7019e-02],
[ 3.9479e-02, -3.5843e-02, 1.2989e-02, -7.2656e-03, 5.7989e-02,
6.7019e-02, 1.3826e-01]], dtype=torch.float64,
grad_fn=<ExpandBackward0>)
learn_Aa36_nc.recorder.plot_loss()torch.save(learn_Aa36_nc.model, "trained_4_feb_All_gap_all_36_no_control_v1.pickle")show_results(learn_Aa36_nc, items=items_Aa36_nc, control_map=control_map)No Control - All variables - 48 all gap
dls_Aa48_nc = imp_dataloader(haiB, hai_eraB, var_sel = list(haiB.columns), block_len=350, gap_len=48, bs=20, control_lags=[1], n_rep=2).cpu()model_Aa48_nc = model_Aa_nc.copy()
model_Aa48_nc.use_control = Falsesave_models_Aa48_nc= SaveModelsBatch(times_epoch=1) # save once per repetitionitems_Aa48_nc = random.choices(dls_Aa48_nc.valid.items, k=4)learn_Aa48_nc = Learner(dls_Aa48_nc, model_Aa48_nc, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_Aa48_nc], metrics=rmse_gap)show_results(learn_Aa48_nc, items=items_Aa48_nc, control_map=control_map)[MeteoImpItem(i=640, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=48), MeteoImpItem(i=534, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=48), MeteoImpItem(i=621, shift=-175, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=48), MeteoImpItem(i=559, shift=-175, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=48)]
learn_Aa48_nc.fit(5, 3e-4)
0.00% [0/5 00:00<?]
| epoch | train_loss | valid_loss | rmse_gap | time |
|---|
0.00% [0/51 00:00<?]
ValueError: Expected parameter covariance_matrix (Tensor of shape (7, 7)) of distribution MultivariateNormal(loc: torch.Size([7]), covariance_matrix: torch.Size([7, 7])) to satisfy the constraint PositiveDefinite(), but found invalid values:
tensor([[-2.1252e+31, -7.2463e+31, -5.0732e+29, -7.1944e+29, 1.8079e+31,
7.5784e+30, -4.2668e+31],
[-7.2463e+31, -2.4719e+32, -1.8432e+30, -2.4103e+30, 6.1574e+31,
2.6057e+31, -1.4537e+32],
[-5.0732e+29, -1.8432e+30, -1.2298e+29, 2.4634e+28, 3.6487e+29,
3.9303e+29, -9.0798e+29],
[-7.1944e+29, -2.4103e+30, 2.4634e+28, -4.0120e+28, 6.3716e+29,
1.7656e+29, -1.4861e+30],
[ 1.8079e+31, 6.1574e+31, 3.6487e+29, 6.3716e+29, -1.5419e+31,
-6.3192e+30, 3.6363e+31],
[ 7.5784e+30, 2.6057e+31, 3.9303e+29, 1.7656e+29, -6.3192e+30,
-3.1082e+30, 1.5004e+31],
[-4.2668e+31, -1.4537e+32, -9.0798e+29, -1.4861e+30, 3.6363e+31,
1.5004e+31, -8.5775e+31]], dtype=torch.float64,
grad_fn=<ExpandBackward0>)
learn_Aa48_nc.recorder.plot_loss()
torch.save(learn_Aa48_nc.model, "trained_4_feb_All_gap_all_48_no_control_v1.pickle")show_results(learn_Aa48_nc, items=items_Aa48_nc, control_map=control_map)[MeteoImpItem(i=3077, shift=-35, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=3195, shift=-35, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=3230, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12), MeteoImpItem(i=2910, shift=0, var_sel=['TA', 'SW_IN', 'VPD', 'PA', 'P', 'WS', 'LW_IN'], gap_len=12)]
LW_IN - gap 12
dls_LW_IN = imp_dataloader(haiB, hai_eraB, var_sel = 'LW_IN', block_len=50, gap_len=12, bs=20, control_lags=[1], n_rep=5).cpu()model_LW_IN = model_Av.copy()save_models_LW_IN = SaveModelsBatch(times_epoch=10)items_LW_IN = random.choices(dls_LW_IN.valid.items, k=4)learn_LW_IN = Learner(dls_LW_IN, model_LW_IN, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_LW_IN], metrics=rmse_gap)learn_LW_IN.fit(1, 1e-3)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 3.196418 | 3.997740 | 0.280681 | 11:39 |
learn_LW_IN.recorder.plot_loss()
torch.save(learn_LW_IN.model, "trained_4_feb_LW_IN_gap_12_v1.pickle")show_results(learn_LW_IN, items = items_LW_IN, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=4462, shift=15, var_sel=['LW_IN'], gap_len=12), MeteoImpItem(i=4415, shift=5, var_sel=['LW_IN'], gap_len=12), MeteoImpItem(i=4322, shift=15, var_sel=['LW_IN'], gap_len=12), MeteoImpItem(i=4212, shift=-5, var_sel=['LW_IN'], gap_len=12)]
LW_IN - 96
after the first trainig fine tune the model for longer gaps
dls_LW_IN96 = imp_dataloader(haiB, hai_eraB, var_sel = 'LW_IN', block_len=300, gap_len=96, bs=20, control_lags=[1], n_rep=5).cpu()model_LW_IN96 = model_LW_IN.copy()save_models_LW_IN96 = SaveModelsBatch(times_epoch=5)items_LW_IN96 = random.choices(dls_LW_IN96.valid.items, k=4)learn_LW_IN96 = Learner(dls_LW_IN96, model_LW_IN96, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models_LW_IN96], metrics=rmse_gap)show_results(learn_LW_IN96, items = items_LW_IN96, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=634, shift=-90, var_sel=['LW_IN'], gap_len=96), MeteoImpItem(i=735, shift=-30, var_sel=['LW_IN'], gap_len=96), MeteoImpItem(i=651, shift=-150, var_sel=['LW_IN'], gap_len=96), MeteoImpItem(i=627, shift=-90, var_sel=['LW_IN'], gap_len=96)]
learn_LW_IN96.fit(1, 1e-4)| epoch | train_loss | valid_loss | rmse_gap | time |
|---|---|---|---|---|
| 0 | 51.693777 | 57.928376 | 0.419843 | 21:42 |
learn_LW_IN96.recorder.plot_loss()
show_results(learn_LW_IN96, items = items_LW_IN96, control_map = control_map, hide_no_gap=True)[MeteoImpItem(i=634, shift=-90, var_sel=['LW_IN'], gap_len=96), MeteoImpItem(i=735, shift=-30, var_sel=['LW_IN'], gap_len=96), MeteoImpItem(i=651, shift=-150, var_sel=['LW_IN'], gap_len=96), MeteoImpItem(i=627, shift=-90, var_sel=['LW_IN'], gap_len=96)]
torch.save(learn_LW_IN96.model, "trained_4_feb_LW_IN_gap_96_v1.pickle")Conditional
dls3 = imp_dataloader(haiB, haiB, var_sel = gen_var_sel(list(haiB.columns)), block_len=100, gap_len=10, bs=20, control_lags=[1], n_rep=10).cpu()model3 = KalmanFilterSR.init_local_slope_pca(len(haiB.columns),len(haiB.columns), df_pca = None, pred_only_gap=True, use_conditional=True)save_models3 = SaveModelsBatch(times_epoch=1) # save once per repetitionitems = random.choices(dls3.valid.items, k=4)learn3 = Learner(dls3, model3, KalmanLoss(only_gap=True), cbs = [Float64Callback, save_models2], metrics=rmse_gap)learn3.fit(20, 1e-4)learn3.recorder.plot_loss()torch.save(learn3.model, "trained_2_feb_gap_partial_var_cond_v1.pickle")# learnB.export("trained_2_feb_gap_all_var_v1")show_results(learn3, items=items)